{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiment: test_restoration\n",
    "\n",
    "Evaluate if restoration affected existing capabilities. Comparing two approaches to calculate coactivations to see if they are getting to the same values.\n",
    "\n",
    "#### Conclusion\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "base_exp_config = dict(\n",
    "    device=\"cuda\",\n",
    "    # dataset related\n",
    "    dataset_name=\"PreprocessedGSC\",\n",
    "    data_dir=\"~/nta/datasets/gsc\",\n",
    "    batch_size_train=(4, 16),\n",
    "    batch_size_test=1000,\n",
    "    # network related\n",
    "    network=tune.grid_search([\"GSCHeb_v0\", \"GSCHeb\"]),\n",
    "    optim_alg=\"SGD\",\n",
    "    momentum=0,  # 0.9,\n",
    "    learning_rate=0.01,  # 0.1,\n",
    "    weight_decay=0.01,  # 1e-4,\n",
    "    lr_scheduler=\"MultiStepLR\",\n",
    "    lr_milestones=[30, 60, 90],\n",
    "    lr_gamma=0.9,  # 0.1,\n",
    "    use_kwinners=True,\n",
    "    # sparse_linear_only=True, # False\n",
    "    # model related\n",
    "    model=\"DSNNWeightedMag\",\n",
    "    # on_perc=0.04,\n",
    "    # sparse related\n",
    "    on_perc=tune.grid_search([0.02, 0.03, 0.04]),\n",
    "    weight_prune_perc=0.3,\n",
    "    # weight_prune_perc=tune.grid_search(list(np.arange(0, 1.001, 0.05))),\n",
    "    # pruning_early_stop=2,\n",
    "    # additional validation\n",
    "    # test_noise=False,\n",
    "    # debugging\n",
    "    # debug_weights=True,\n",
    "    # debug_sparse=False,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import tabulate\n",
    "import pprint\n",
    "import click\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ray.tune.commands import *\n",
    "from nupic.research.frameworks.dynamic_sparse.common.browser import *\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rcParams\n",
    "\n",
    "%config InlineBackend.figure_format = 'retina'\n",
    "\n",
    "import seaborn as sns\n",
    "sns.set(style=\"whitegrid\")\n",
    "sns.set_palette(\"colorblind\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and check data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "exps = ['test_restoration_5']\n",
    "paths = [os.path.expanduser(\"~/nta/results/{}\".format(e)) for e in exps]\n",
    "df = load_many(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>train_acc_max</th>\n",
       "      <th>train_acc_max_epoch</th>\n",
       "      <th>train_acc_min</th>\n",
       "      <th>train_acc_min_epoch</th>\n",
       "      <th>train_acc_median</th>\n",
       "      <th>train_acc_last</th>\n",
       "      <th>val_acc_max</th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th>val_acc_min</th>\n",
       "      <th>...</th>\n",
       "      <th>lr_milestones</th>\n",
       "      <th>lr_scheduler</th>\n",
       "      <th>model</th>\n",
       "      <th>momentum</th>\n",
       "      <th>network</th>\n",
       "      <th>on_perc</th>\n",
       "      <th>optim_alg</th>\n",
       "      <th>use_kwinners</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>weight_prune_perc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0_network=GSCHeb_v0,on_perc=0.02</td>\n",
       "      <td>0.780295</td>\n",
       "      <td>62</td>\n",
       "      <td>0.248023</td>\n",
       "      <td>0</td>\n",
       "      <td>0.749780</td>\n",
       "      <td>0.733913</td>\n",
       "      <td>0.865277</td>\n",
       "      <td>63</td>\n",
       "      <td>0.138332</td>\n",
       "      <td>...</td>\n",
       "      <td>60.0</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>DSNNWeightedMag</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb_v0</td>\n",
       "      <td>0.02</td>\n",
       "      <td>SGD</td>\n",
       "      <td>True</td>\n",
       "      <td>0.01</td>\n",
       "      <td>None-None-0.3-0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1_network=GSCHeb,on_perc=0.02</td>\n",
       "      <td>0.811298</td>\n",
       "      <td>46</td>\n",
       "      <td>0.241724</td>\n",
       "      <td>0</td>\n",
       "      <td>0.779709</td>\n",
       "      <td>0.766868</td>\n",
       "      <td>0.884523</td>\n",
       "      <td>19</td>\n",
       "      <td>0.309944</td>\n",
       "      <td>...</td>\n",
       "      <td>60.0</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>DSNNWeightedMag</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb</td>\n",
       "      <td>0.02</td>\n",
       "      <td>SGD</td>\n",
       "      <td>True</td>\n",
       "      <td>0.01</td>\n",
       "      <td>None-None-0.3-0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2_network=GSCHeb_v0,on_perc=0.03</td>\n",
       "      <td>0.854799</td>\n",
       "      <td>60</td>\n",
       "      <td>0.263548</td>\n",
       "      <td>0</td>\n",
       "      <td>0.842301</td>\n",
       "      <td>0.839566</td>\n",
       "      <td>0.921010</td>\n",
       "      <td>60</td>\n",
       "      <td>0.384122</td>\n",
       "      <td>...</td>\n",
       "      <td>60.0</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>DSNNWeightedMag</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb_v0</td>\n",
       "      <td>0.03</td>\n",
       "      <td>SGD</td>\n",
       "      <td>True</td>\n",
       "      <td>0.01</td>\n",
       "      <td>None-None-0.3-0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3_network=GSCHeb,on_perc=0.03</td>\n",
       "      <td>0.855239</td>\n",
       "      <td>70</td>\n",
       "      <td>0.263939</td>\n",
       "      <td>0</td>\n",
       "      <td>0.837394</td>\n",
       "      <td>0.846988</td>\n",
       "      <td>0.923817</td>\n",
       "      <td>96</td>\n",
       "      <td>0.141540</td>\n",
       "      <td>...</td>\n",
       "      <td>60.0</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>DSNNWeightedMag</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb</td>\n",
       "      <td>0.03</td>\n",
       "      <td>SGD</td>\n",
       "      <td>True</td>\n",
       "      <td>0.01</td>\n",
       "      <td>None-None-0.3-0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4_network=GSCHeb_v0,on_perc=0.04</td>\n",
       "      <td>0.864515</td>\n",
       "      <td>91</td>\n",
       "      <td>0.281564</td>\n",
       "      <td>0</td>\n",
       "      <td>0.851723</td>\n",
       "      <td>0.859438</td>\n",
       "      <td>0.925421</td>\n",
       "      <td>65</td>\n",
       "      <td>0.336808</td>\n",
       "      <td>...</td>\n",
       "      <td>60.0</td>\n",
       "      <td>MultiStepLR</td>\n",
       "      <td>DSNNWeightedMag</td>\n",
       "      <td>0</td>\n",
       "      <td>GSCHeb_v0</td>\n",
       "      <td>0.04</td>\n",
       "      <td>SGD</td>\n",
       "      <td>True</td>\n",
       "      <td>0.01</td>\n",
       "      <td>None-None-0.3-0.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Experiment Name  train_acc_max  train_acc_max_epoch  \\\n",
       "0  0_network=GSCHeb_v0,on_perc=0.02       0.780295                   62   \n",
       "1     1_network=GSCHeb,on_perc=0.02       0.811298                   46   \n",
       "2  2_network=GSCHeb_v0,on_perc=0.03       0.854799                   60   \n",
       "3     3_network=GSCHeb,on_perc=0.03       0.855239                   70   \n",
       "4  4_network=GSCHeb_v0,on_perc=0.04       0.864515                   91   \n",
       "\n",
       "   train_acc_min  train_acc_min_epoch  train_acc_median  train_acc_last  \\\n",
       "0       0.248023                    0          0.749780        0.733913   \n",
       "1       0.241724                    0          0.779709        0.766868   \n",
       "2       0.263548                    0          0.842301        0.839566   \n",
       "3       0.263939                    0          0.837394        0.846988   \n",
       "4       0.281564                    0          0.851723        0.859438   \n",
       "\n",
       "   val_acc_max  val_acc_max_epoch  val_acc_min  ...  lr_milestones  \\\n",
       "0     0.865277                 63     0.138332  ...           60.0   \n",
       "1     0.884523                 19     0.309944  ...           60.0   \n",
       "2     0.921010                 60     0.384122  ...           60.0   \n",
       "3     0.923817                 96     0.141540  ...           60.0   \n",
       "4     0.925421                 65     0.336808  ...           60.0   \n",
       "\n",
       "   lr_scheduler            model momentum    network on_perc  optim_alg  \\\n",
       "0   MultiStepLR  DSNNWeightedMag        0  GSCHeb_v0    0.02        SGD   \n",
       "1   MultiStepLR  DSNNWeightedMag        0     GSCHeb    0.02        SGD   \n",
       "2   MultiStepLR  DSNNWeightedMag        0  GSCHeb_v0    0.03        SGD   \n",
       "3   MultiStepLR  DSNNWeightedMag        0     GSCHeb    0.03        SGD   \n",
       "4   MultiStepLR  DSNNWeightedMag        0  GSCHeb_v0    0.04        SGD   \n",
       "\n",
       "   use_kwinners  weight_decay  weight_prune_perc  \n",
       "0          True          0.01  None-None-0.3-0.3  \n",
       "1          True          0.01  None-None-0.3-0.3  \n",
       "2          True          0.01  None-None-0.3-0.3  \n",
       "3          True          0.01  None-None-0.3-0.3  \n",
       "4          True          0.01  None-None-0.3-0.3  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace hebbian prine\n",
    "# df['hebbian_prune_perc'] = df['hebbian_prune_perc'].replace(np.nan, 0.0, regex=True)\n",
    "# df['weight_prune_perc'] = df['weight_prune_perc'].replace(np.nan, 0.0, regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Experiment Name', 'train_acc_max', 'train_acc_max_epoch',\n",
       "       'train_acc_min', 'train_acc_min_epoch', 'train_acc_median',\n",
       "       'train_acc_last', 'val_acc_max', 'val_acc_max_epoch', 'val_acc_min',\n",
       "       'val_acc_min_epoch', 'val_acc_median', 'val_acc_last', 'val_acc_all',\n",
       "       'epochs', 'experiment_file_name', 'trial_time', 'mean_epoch_time',\n",
       "       'batch_size_test', 'batch_size_train', 'data_dir', 'dataset_name',\n",
       "       'device', 'learning_rate', 'lr_gamma', 'lr_milestones', 'lr_scheduler',\n",
       "       'model', 'momentum', 'network', 'on_perc', 'optim_alg', 'use_kwinners',\n",
       "       'weight_decay', 'weight_prune_perc'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60, 35)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Experiment Name                             1_network=GSCHeb,on_perc=0.02\n",
       "train_acc_max                                                    0.811298\n",
       "train_acc_max_epoch                                                    46\n",
       "train_acc_min                                                    0.241724\n",
       "train_acc_min_epoch                                                     0\n",
       "train_acc_median                                                 0.779709\n",
       "train_acc_last                                                   0.766868\n",
       "val_acc_max                                                      0.884523\n",
       "val_acc_max_epoch                                                      19\n",
       "val_acc_min                                                      0.309944\n",
       "val_acc_min_epoch                                                      81\n",
       "val_acc_median                                                   0.846231\n",
       "val_acc_last                                                     0.836006\n",
       "val_acc_all             0     0.377706\n",
       "1     0.630714\n",
       "2     0.695269\n",
       "3...\n",
       "epochs                                                                100\n",
       "experiment_file_name    /Users/lsouza/nta/results/test_restoration_5/e...\n",
       "trial_time                                                        33.2763\n",
       "mean_epoch_time                                                  0.332763\n",
       "batch_size_test                                                      1000\n",
       "batch_size_train                                                       10\n",
       "data_dir                                               ~/nta/datasets/gsc\n",
       "dataset_name                                              PreprocessedGSC\n",
       "device                                                               cuda\n",
       "learning_rate                                                        0.01\n",
       "lr_gamma                                                              0.9\n",
       "lr_milestones                                                          60\n",
       "lr_scheduler                                                  MultiStepLR\n",
       "model                                                     DSNNWeightedMag\n",
       "momentum                                                                0\n",
       "network                                                            GSCHeb\n",
       "on_perc                                                              0.02\n",
       "optim_alg                                                             SGD\n",
       "use_kwinners                                                         True\n",
       "weight_decay                                                         0.01\n",
       "weight_prune_perc                                       None-None-0.3-0.3\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model\n",
       "DSNNWeightedMag    60\n",
       "Name: model, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('model')['model'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ## Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiment Details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Did any  trials failed?\n",
    "df[df[\"epochs\"]<num_epochs][\"epochs\"].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60, 35)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing failed or incomplete trials\n",
    "df_origin = df.copy()\n",
    "df = df_origin[df_origin[\"epochs\"]>=num_epochs]\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: epochs, dtype: int64)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# which ones failed?\n",
    "# failed, or still ongoing?\n",
    "df_origin['failed'] = df_origin[\"epochs\"]<num_epochs\n",
    "df_origin[df_origin['failed']]['epochs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper functions\n",
    "def mean_and_std(s):\n",
    "    return \"{:.3f} ± {:.3f}\".format(s.mean(), s.std())\n",
    "\n",
    "def round_mean(s):\n",
    "    return \"{:.0f}\".format(round(s.mean()))\n",
    "\n",
    "stats = ['min', 'max', 'mean', 'std']\n",
    "\n",
    "def agg(columns, filter=None, round=3):\n",
    "    if filter is None:\n",
    "        return (df.groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n",
    "    else:\n",
    "        return (df[filter].groupby(columns)\n",
    "             .agg({'val_acc_max_epoch': round_mean,\n",
    "                   'val_acc_max': stats,                \n",
    "                   'model': ['count']})).round(round)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Does improved weight pruning outperforms regular SET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>val_acc_max_epoch</th>\n",
       "      <th colspan=\"4\" halign=\"left\">val_acc_max</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>round_mean</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>on_perc</th>\n",
       "      <th>network</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.02</th>\n",
       "      <th>GSCHeb</th>\n",
       "      <td>37</td>\n",
       "      <td>0.864</td>\n",
       "      <td>0.889</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.008</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSCHeb_v0</th>\n",
       "      <td>42</td>\n",
       "      <td>0.865</td>\n",
       "      <td>0.887</td>\n",
       "      <td>0.876</td>\n",
       "      <td>0.006</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.03</th>\n",
       "      <th>GSCHeb</th>\n",
       "      <td>66</td>\n",
       "      <td>0.893</td>\n",
       "      <td>0.924</td>\n",
       "      <td>0.914</td>\n",
       "      <td>0.010</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSCHeb_v0</th>\n",
       "      <td>62</td>\n",
       "      <td>0.907</td>\n",
       "      <td>0.922</td>\n",
       "      <td>0.916</td>\n",
       "      <td>0.005</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0.04</th>\n",
       "      <th>GSCHeb</th>\n",
       "      <td>81</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.933</td>\n",
       "      <td>0.004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GSCHeb_v0</th>\n",
       "      <td>73</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.938</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.004</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  val_acc_max_epoch val_acc_max                      model\n",
       "                         round_mean         min    max   mean    std count\n",
       "on_perc network                                                           \n",
       "0.02    GSCHeb                   37       0.864  0.889  0.875  0.008    10\n",
       "        GSCHeb_v0                42       0.865  0.887  0.876  0.006    10\n",
       "0.03    GSCHeb                   66       0.893  0.924  0.914  0.010    10\n",
       "        GSCHeb_v0                62       0.907  0.922  0.916  0.005    10\n",
       "0.04    GSCHeb                   81       0.923  0.937  0.933  0.004    10\n",
       "        GSCHeb_v0                73       0.923  0.938  0.930  0.004    10"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg(['on_perc', 'network'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
